{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "mobilenet_on_sign_language_digits.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1cicugkTaywIz03rkOMWDSjq0OcEvTCfF",
      "authorship_tag": "ABX9TyNjBkpIA6FzzFd/KF2SbduG",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SummerLife/EmbeddedSystem/blob/master/MachineLearning/gist/mobilenet_on_sign_language_digits.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pzq043ncqbrj",
        "colab_type": "text"
      },
      "source": [
        "## Building A Fine-Tuned MobileNet Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-75DJsOSnkUM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.layers import Dense, Activation\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "from tensorflow.keras.metrics import categorical_crossentropy\n",
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
        "from tensorflow.keras.preprocessing import image\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.applications import imagenet_utils\n",
        "from sklearn.metrics import confusion_matrix\n",
        "import itertools\n",
        "import os\n",
        "import shutil\n",
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "22aWbEwwnvbB",
        "colab_type": "text"
      },
      "source": [
        "## Process The Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yp0Mo7rcAvFA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data_set_dir = \"/content/sign_language_digits_dataset\"\n",
        "\n",
        "# Sign Language Digits data set:\n",
        "# https://github.com/ardamavi/Sign-Language-Digits-Dataset\n",
        "\n",
        "# Organize data into train, valid, test dirs\n",
        "# os.chdir(data_set_dir)\n",
        "# if os.path.isdir('train/0/') is False: \n",
        "#     os.mkdir('train')\n",
        "#     os.mkdir('valid')\n",
        "#     os.mkdir('test')\n",
        "\n",
        "#     for i in range(0, 10):\n",
        "#         shutil.move(f'{i}', 'train')\n",
        "#         os.mkdir(f'valid/{i}')\n",
        "#         os.mkdir(f'test/{i}')\n",
        "\n",
        "#         valid_samples = random.sample(os.listdir(f'train/{i}'), 30)\n",
        "#         for j in valid_samples:\n",
        "#             shutil.move(f'train/{i}/{j}', f'valid/{i}')\n",
        "\n",
        "#         test_samples = random.sample(os.listdir(f'train/{i}'), 5)\n",
        "#         for k in test_samples:\n",
        "#             shutil.move(f'train/{i}/{k}', f'test/{i}')\n",
        "# os.chdir('../..')"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w3iKSL2GmcWi",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "d4a49922-f1d9-4319-b973-c5bbb4c9a02e"
      },
      "source": [
        "train_path = data_set_dir + '/train'\n",
        "valid_path = data_set_dir + '/valid'\n",
        "test_path  = data_set_dir + '/test'\n",
        "\n",
        "train_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input).flow_from_directory(\n",
        "    directory=train_path, target_size=(224,224), batch_size=10)\n",
        "valid_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input).flow_from_directory(\n",
        "    directory=valid_path, target_size=(224,224), batch_size=10)\n",
        "test_batches = ImageDataGenerator(preprocessing_function=tf.keras.applications.mobilenet.preprocess_input).flow_from_directory(\n",
        "    directory=test_path, target_size=(224,224), batch_size=10, shuffle=False)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 1712 images belonging to 10 classes.\n",
            "Found 300 images belonging to 10 classes.\n",
            "Found 50 images belonging to 10 classes.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hathsyugnyI6",
        "colab_type": "text"
      },
      "source": [
        "## Fine-Tuning MobileNet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5akka0JenZZm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3c39d400-1e60-4dfa-f76e-7004ff89e864"
      },
      "source": [
        "mobile = tf.keras.applications.mobilenet.MobileNet()\n",
        "mobile.summary()"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"mobilenet_1.00_224\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "conv1_pad (ZeroPadding2D)    (None, 225, 225, 3)       0         \n",
            "_________________________________________________________________\n",
            "conv1 (Conv2D)               (None, 112, 112, 32)      864       \n",
            "_________________________________________________________________\n",
            "conv1_bn (BatchNormalization (None, 112, 112, 32)      128       \n",
            "_________________________________________________________________\n",
            "conv1_relu (ReLU)            (None, 112, 112, 32)      0         \n",
            "_________________________________________________________________\n",
            "conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 32)      288       \n",
            "_________________________________________________________________\n",
            "conv_dw_1_bn (BatchNormaliza (None, 112, 112, 32)      128       \n",
            "_________________________________________________________________\n",
            "conv_dw_1_relu (ReLU)        (None, 112, 112, 32)      0         \n",
            "_________________________________________________________________\n",
            "conv_pw_1 (Conv2D)           (None, 112, 112, 64)      2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_1_bn (BatchNormaliza (None, 112, 112, 64)      256       \n",
            "_________________________________________________________________\n",
            "conv_pw_1_relu (ReLU)        (None, 112, 112, 64)      0         \n",
            "_________________________________________________________________\n",
            "conv_pad_2 (ZeroPadding2D)   (None, 113, 113, 64)      0         \n",
            "_________________________________________________________________\n",
            "conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 64)        576       \n",
            "_________________________________________________________________\n",
            "conv_dw_2_bn (BatchNormaliza (None, 56, 56, 64)        256       \n",
            "_________________________________________________________________\n",
            "conv_dw_2_relu (ReLU)        (None, 56, 56, 64)        0         \n",
            "_________________________________________________________________\n",
            "conv_pw_2 (Conv2D)           (None, 56, 56, 128)       8192      \n",
            "_________________________________________________________________\n",
            "conv_pw_2_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_pw_2_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 128)       1152      \n",
            "_________________________________________________________________\n",
            "conv_dw_3_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_dw_3_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_3 (Conv2D)           (None, 56, 56, 128)       16384     \n",
            "_________________________________________________________________\n",
            "conv_pw_3_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_pw_3_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_4 (ZeroPadding2D)   (None, 57, 57, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 128)       1152      \n",
            "_________________________________________________________________\n",
            "conv_dw_4_bn (BatchNormaliza (None, 28, 28, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_dw_4_relu (ReLU)        (None, 28, 28, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_4 (Conv2D)           (None, 28, 28, 256)       32768     \n",
            "_________________________________________________________________\n",
            "conv_pw_4_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_pw_4_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 256)       2304      \n",
            "_________________________________________________________________\n",
            "conv_dw_5_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_dw_5_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_5 (Conv2D)           (None, 28, 28, 256)       65536     \n",
            "_________________________________________________________________\n",
            "conv_pw_5_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_pw_5_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_6 (ZeroPadding2D)   (None, 29, 29, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 256)       2304      \n",
            "_________________________________________________________________\n",
            "conv_dw_6_bn (BatchNormaliza (None, 14, 14, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_dw_6_relu (ReLU)        (None, 14, 14, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_6 (Conv2D)           (None, 14, 14, 512)       131072    \n",
            "_________________________________________________________________\n",
            "conv_pw_6_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_6_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_7_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_7_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_7 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_7_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_7_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_8_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_8_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_8 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_8_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_8_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_9_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_9_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_9 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_9_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_9_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_10 (DepthwiseConv2D) (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_10_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_10_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_10 (Conv2D)          (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_10_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_10_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_11 (DepthwiseConv2D) (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_11_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_11_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_11 (Conv2D)          (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_11_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_11_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_12 (DepthwiseConv2D) (None, 7, 7, 512)         4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_12_bn (BatchNormaliz (None, 7, 7, 512)         2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_12_relu (ReLU)       (None, 7, 7, 512)         0         \n",
            "_________________________________________________________________\n",
            "conv_pw_12 (Conv2D)          (None, 7, 7, 1024)        524288    \n",
            "_________________________________________________________________\n",
            "conv_pw_12_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_pw_12_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv_dw_13 (DepthwiseConv2D) (None, 7, 7, 1024)        9216      \n",
            "_________________________________________________________________\n",
            "conv_dw_13_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_dw_13_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv_pw_13 (Conv2D)          (None, 7, 7, 1024)        1048576   \n",
            "_________________________________________________________________\n",
            "conv_pw_13_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_pw_13_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "global_average_pooling2d_1 ( (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "reshape_1 (Reshape)          (None, 1, 1, 1024)        0         \n",
            "_________________________________________________________________\n",
            "dropout (Dropout)            (None, 1, 1, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv_preds (Conv2D)          (None, 1, 1, 1000)        1025000   \n",
            "_________________________________________________________________\n",
            "reshape_2 (Reshape)          (None, 1000)              0         \n",
            "_________________________________________________________________\n",
            "predictions (Activation)     (None, 1000)              0         \n",
            "=================================================================\n",
            "Total params: 4,253,864\n",
            "Trainable params: 4,231,976\n",
            "Non-trainable params: 21,888\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mUv736VOn-K3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e9aca223-7991-4282-807a-2283ec0aab84"
      },
      "source": [
        "# grab the output from the sixth to last layer of the model and store it in this variable x.\n",
        "x = mobile.layers[-6].output\n",
        "print(x)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Tensor(\"global_average_pooling2d_1/Identity:0\", shape=(None, 1024), dtype=float32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pfOawbjdoNVi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "output = Dense(units=10, activation='softmax')(x)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D-_g3wSbolvz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "ca390a04-5222-4015-d6eb-62ac25a145ca"
      },
      "source": [
        "model = Model(inputs=mobile.input, outputs=output)\n",
        "model.summary()"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "conv1_pad (ZeroPadding2D)    (None, 225, 225, 3)       0         \n",
            "_________________________________________________________________\n",
            "conv1 (Conv2D)               (None, 112, 112, 32)      864       \n",
            "_________________________________________________________________\n",
            "conv1_bn (BatchNormalization (None, 112, 112, 32)      128       \n",
            "_________________________________________________________________\n",
            "conv1_relu (ReLU)            (None, 112, 112, 32)      0         \n",
            "_________________________________________________________________\n",
            "conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 32)      288       \n",
            "_________________________________________________________________\n",
            "conv_dw_1_bn (BatchNormaliza (None, 112, 112, 32)      128       \n",
            "_________________________________________________________________\n",
            "conv_dw_1_relu (ReLU)        (None, 112, 112, 32)      0         \n",
            "_________________________________________________________________\n",
            "conv_pw_1 (Conv2D)           (None, 112, 112, 64)      2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_1_bn (BatchNormaliza (None, 112, 112, 64)      256       \n",
            "_________________________________________________________________\n",
            "conv_pw_1_relu (ReLU)        (None, 112, 112, 64)      0         \n",
            "_________________________________________________________________\n",
            "conv_pad_2 (ZeroPadding2D)   (None, 113, 113, 64)      0         \n",
            "_________________________________________________________________\n",
            "conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 64)        576       \n",
            "_________________________________________________________________\n",
            "conv_dw_2_bn (BatchNormaliza (None, 56, 56, 64)        256       \n",
            "_________________________________________________________________\n",
            "conv_dw_2_relu (ReLU)        (None, 56, 56, 64)        0         \n",
            "_________________________________________________________________\n",
            "conv_pw_2 (Conv2D)           (None, 56, 56, 128)       8192      \n",
            "_________________________________________________________________\n",
            "conv_pw_2_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_pw_2_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 128)       1152      \n",
            "_________________________________________________________________\n",
            "conv_dw_3_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_dw_3_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_3 (Conv2D)           (None, 56, 56, 128)       16384     \n",
            "_________________________________________________________________\n",
            "conv_pw_3_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_pw_3_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_4 (ZeroPadding2D)   (None, 57, 57, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 128)       1152      \n",
            "_________________________________________________________________\n",
            "conv_dw_4_bn (BatchNormaliza (None, 28, 28, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_dw_4_relu (ReLU)        (None, 28, 28, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_4 (Conv2D)           (None, 28, 28, 256)       32768     \n",
            "_________________________________________________________________\n",
            "conv_pw_4_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_pw_4_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 256)       2304      \n",
            "_________________________________________________________________\n",
            "conv_dw_5_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_dw_5_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_5 (Conv2D)           (None, 28, 28, 256)       65536     \n",
            "_________________________________________________________________\n",
            "conv_pw_5_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_pw_5_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_6 (ZeroPadding2D)   (None, 29, 29, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 256)       2304      \n",
            "_________________________________________________________________\n",
            "conv_dw_6_bn (BatchNormaliza (None, 14, 14, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_dw_6_relu (ReLU)        (None, 14, 14, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_6 (Conv2D)           (None, 14, 14, 512)       131072    \n",
            "_________________________________________________________________\n",
            "conv_pw_6_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_6_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_7_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_7_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_7 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_7_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_7_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_8_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_8_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_8 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_8_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_8_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_9_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_9_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_9 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_9_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_9_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_10 (DepthwiseConv2D) (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_10_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_10_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_10 (Conv2D)          (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_10_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_10_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_11 (DepthwiseConv2D) (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_11_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_11_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_11 (Conv2D)          (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_11_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_11_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_12 (DepthwiseConv2D) (None, 7, 7, 512)         4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_12_bn (BatchNormaliz (None, 7, 7, 512)         2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_12_relu (ReLU)       (None, 7, 7, 512)         0         \n",
            "_________________________________________________________________\n",
            "conv_pw_12 (Conv2D)          (None, 7, 7, 1024)        524288    \n",
            "_________________________________________________________________\n",
            "conv_pw_12_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_pw_12_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv_dw_13 (DepthwiseConv2D) (None, 7, 7, 1024)        9216      \n",
            "_________________________________________________________________\n",
            "conv_dw_13_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_dw_13_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv_pw_13 (Conv2D)          (None, 7, 7, 1024)        1048576   \n",
            "_________________________________________________________________\n",
            "conv_pw_13_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_pw_13_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "global_average_pooling2d_1 ( (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 10)                10250     \n",
            "=================================================================\n",
            "Total params: 3,239,114\n",
            "Trainable params: 3,217,226\n",
            "Non-trainable params: 21,888\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mZ-pfSGGqW0s",
        "colab_type": "text"
      },
      "source": [
        "we need to choose how many layers we actually want to be trained when we train on our new data set."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "clvbDfmkqUjt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "for layer in model.layers[:-23]:\n",
        "    layer.trainable = False"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xhJbcVfGqp3S",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "67371aa0-c06e-4e09-9232-f91664b80d44"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_2 (InputLayer)         [(None, 224, 224, 3)]     0         \n",
            "_________________________________________________________________\n",
            "conv1_pad (ZeroPadding2D)    (None, 225, 225, 3)       0         \n",
            "_________________________________________________________________\n",
            "conv1 (Conv2D)               (None, 112, 112, 32)      864       \n",
            "_________________________________________________________________\n",
            "conv1_bn (BatchNormalization (None, 112, 112, 32)      128       \n",
            "_________________________________________________________________\n",
            "conv1_relu (ReLU)            (None, 112, 112, 32)      0         \n",
            "_________________________________________________________________\n",
            "conv_dw_1 (DepthwiseConv2D)  (None, 112, 112, 32)      288       \n",
            "_________________________________________________________________\n",
            "conv_dw_1_bn (BatchNormaliza (None, 112, 112, 32)      128       \n",
            "_________________________________________________________________\n",
            "conv_dw_1_relu (ReLU)        (None, 112, 112, 32)      0         \n",
            "_________________________________________________________________\n",
            "conv_pw_1 (Conv2D)           (None, 112, 112, 64)      2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_1_bn (BatchNormaliza (None, 112, 112, 64)      256       \n",
            "_________________________________________________________________\n",
            "conv_pw_1_relu (ReLU)        (None, 112, 112, 64)      0         \n",
            "_________________________________________________________________\n",
            "conv_pad_2 (ZeroPadding2D)   (None, 113, 113, 64)      0         \n",
            "_________________________________________________________________\n",
            "conv_dw_2 (DepthwiseConv2D)  (None, 56, 56, 64)        576       \n",
            "_________________________________________________________________\n",
            "conv_dw_2_bn (BatchNormaliza (None, 56, 56, 64)        256       \n",
            "_________________________________________________________________\n",
            "conv_dw_2_relu (ReLU)        (None, 56, 56, 64)        0         \n",
            "_________________________________________________________________\n",
            "conv_pw_2 (Conv2D)           (None, 56, 56, 128)       8192      \n",
            "_________________________________________________________________\n",
            "conv_pw_2_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_pw_2_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_3 (DepthwiseConv2D)  (None, 56, 56, 128)       1152      \n",
            "_________________________________________________________________\n",
            "conv_dw_3_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_dw_3_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_3 (Conv2D)           (None, 56, 56, 128)       16384     \n",
            "_________________________________________________________________\n",
            "conv_pw_3_bn (BatchNormaliza (None, 56, 56, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_pw_3_relu (ReLU)        (None, 56, 56, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_4 (ZeroPadding2D)   (None, 57, 57, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_4 (DepthwiseConv2D)  (None, 28, 28, 128)       1152      \n",
            "_________________________________________________________________\n",
            "conv_dw_4_bn (BatchNormaliza (None, 28, 28, 128)       512       \n",
            "_________________________________________________________________\n",
            "conv_dw_4_relu (ReLU)        (None, 28, 28, 128)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_4 (Conv2D)           (None, 28, 28, 256)       32768     \n",
            "_________________________________________________________________\n",
            "conv_pw_4_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_pw_4_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_5 (DepthwiseConv2D)  (None, 28, 28, 256)       2304      \n",
            "_________________________________________________________________\n",
            "conv_dw_5_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_dw_5_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_5 (Conv2D)           (None, 28, 28, 256)       65536     \n",
            "_________________________________________________________________\n",
            "conv_pw_5_bn (BatchNormaliza (None, 28, 28, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_pw_5_relu (ReLU)        (None, 28, 28, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_6 (ZeroPadding2D)   (None, 29, 29, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_6 (DepthwiseConv2D)  (None, 14, 14, 256)       2304      \n",
            "_________________________________________________________________\n",
            "conv_dw_6_bn (BatchNormaliza (None, 14, 14, 256)       1024      \n",
            "_________________________________________________________________\n",
            "conv_dw_6_relu (ReLU)        (None, 14, 14, 256)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_6 (Conv2D)           (None, 14, 14, 512)       131072    \n",
            "_________________________________________________________________\n",
            "conv_pw_6_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_6_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_7 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_7_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_7_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_7 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_7_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_7_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_8 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_8_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_8_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_8 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_8_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_8_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_9 (DepthwiseConv2D)  (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_9_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_9_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_9 (Conv2D)           (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_9_bn (BatchNormaliza (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_9_relu (ReLU)        (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_10 (DepthwiseConv2D) (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_10_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_10_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_10 (Conv2D)          (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_10_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_10_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_11 (DepthwiseConv2D) (None, 14, 14, 512)       4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_11_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_11_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pw_11 (Conv2D)          (None, 14, 14, 512)       262144    \n",
            "_________________________________________________________________\n",
            "conv_pw_11_bn (BatchNormaliz (None, 14, 14, 512)       2048      \n",
            "_________________________________________________________________\n",
            "conv_pw_11_relu (ReLU)       (None, 14, 14, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_pad_12 (ZeroPadding2D)  (None, 15, 15, 512)       0         \n",
            "_________________________________________________________________\n",
            "conv_dw_12 (DepthwiseConv2D) (None, 7, 7, 512)         4608      \n",
            "_________________________________________________________________\n",
            "conv_dw_12_bn (BatchNormaliz (None, 7, 7, 512)         2048      \n",
            "_________________________________________________________________\n",
            "conv_dw_12_relu (ReLU)       (None, 7, 7, 512)         0         \n",
            "_________________________________________________________________\n",
            "conv_pw_12 (Conv2D)          (None, 7, 7, 1024)        524288    \n",
            "_________________________________________________________________\n",
            "conv_pw_12_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_pw_12_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv_dw_13 (DepthwiseConv2D) (None, 7, 7, 1024)        9216      \n",
            "_________________________________________________________________\n",
            "conv_dw_13_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_dw_13_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "conv_pw_13 (Conv2D)          (None, 7, 7, 1024)        1048576   \n",
            "_________________________________________________________________\n",
            "conv_pw_13_bn (BatchNormaliz (None, 7, 7, 1024)        4096      \n",
            "_________________________________________________________________\n",
            "conv_pw_13_relu (ReLU)       (None, 7, 7, 1024)        0         \n",
            "_________________________________________________________________\n",
            "global_average_pooling2d_1 ( (None, 1024)              0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 10)                10250     \n",
            "=================================================================\n",
            "Total params: 3,239,114\n",
            "Trainable params: 1,873,930\n",
            "Non-trainable params: 1,365,184\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-vS-dKBDrLGL",
        "colab_type": "text"
      },
      "source": [
        "## Training The Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i4g4jaubq-2u",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(optimizer=Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hRSYT8DLrPdw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "74a4d446-a0f0-4cbd-dd1e-3a5b5507bf3a"
      },
      "source": [
        "model.fit(x=train_batches, validation_data=valid_batches, epochs=30, verbose=2)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/30\n",
            "172/172 - 5s - loss: 0.6319 - accuracy: 0.8160 - val_loss: 0.4088 - val_accuracy: 0.8767\n",
            "Epoch 2/30\n",
            "172/172 - 5s - loss: 0.1238 - accuracy: 0.9796 - val_loss: 0.1200 - val_accuracy: 0.9633\n",
            "Epoch 3/30\n",
            "172/172 - 5s - loss: 0.0562 - accuracy: 0.9959 - val_loss: 0.0720 - val_accuracy: 0.9833\n",
            "Epoch 4/30\n",
            "172/172 - 5s - loss: 0.0297 - accuracy: 0.9988 - val_loss: 0.0679 - val_accuracy: 0.9767\n",
            "Epoch 5/30\n",
            "172/172 - 5s - loss: 0.0295 - accuracy: 0.9959 - val_loss: 0.0852 - val_accuracy: 0.9667\n",
            "Epoch 6/30\n",
            "172/172 - 5s - loss: 0.0151 - accuracy: 0.9994 - val_loss: 0.0347 - val_accuracy: 0.9933\n",
            "Epoch 7/30\n",
            "172/172 - 5s - loss: 0.0115 - accuracy: 0.9994 - val_loss: 0.0316 - val_accuracy: 0.9933\n",
            "Epoch 8/30\n",
            "172/172 - 5s - loss: 0.0123 - accuracy: 0.9988 - val_loss: 0.0430 - val_accuracy: 0.9867\n",
            "Epoch 9/30\n",
            "172/172 - 5s - loss: 0.0111 - accuracy: 0.9988 - val_loss: 0.0340 - val_accuracy: 0.9967\n",
            "Epoch 10/30\n",
            "172/172 - 5s - loss: 0.0083 - accuracy: 0.9994 - val_loss: 0.0262 - val_accuracy: 0.9933\n",
            "Epoch 11/30\n",
            "172/172 - 5s - loss: 0.0062 - accuracy: 0.9994 - val_loss: 0.0187 - val_accuracy: 1.0000\n",
            "Epoch 12/30\n",
            "172/172 - 5s - loss: 0.0055 - accuracy: 1.0000 - val_loss: 0.0163 - val_accuracy: 0.9967\n",
            "Epoch 13/30\n",
            "172/172 - 5s - loss: 0.0056 - accuracy: 0.9994 - val_loss: 0.0394 - val_accuracy: 0.9900\n",
            "Epoch 14/30\n",
            "172/172 - 5s - loss: 0.0043 - accuracy: 1.0000 - val_loss: 0.0163 - val_accuracy: 1.0000\n",
            "Epoch 15/30\n",
            "172/172 - 5s - loss: 0.0048 - accuracy: 1.0000 - val_loss: 0.0501 - val_accuracy: 0.9767\n",
            "Epoch 16/30\n",
            "172/172 - 5s - loss: 0.0031 - accuracy: 1.0000 - val_loss: 0.0217 - val_accuracy: 0.9933\n",
            "Epoch 17/30\n",
            "172/172 - 5s - loss: 0.0058 - accuracy: 0.9988 - val_loss: 0.0153 - val_accuracy: 0.9967\n",
            "Epoch 18/30\n",
            "172/172 - 5s - loss: 0.0022 - accuracy: 1.0000 - val_loss: 0.0094 - val_accuracy: 1.0000\n",
            "Epoch 19/30\n",
            "172/172 - 5s - loss: 0.0020 - accuracy: 1.0000 - val_loss: 0.0148 - val_accuracy: 0.9967\n",
            "Epoch 20/30\n",
            "172/172 - 5s - loss: 0.0118 - accuracy: 0.9982 - val_loss: 0.0729 - val_accuracy: 0.9800\n",
            "Epoch 21/30\n",
            "172/172 - 5s - loss: 0.0057 - accuracy: 0.9988 - val_loss: 0.0400 - val_accuracy: 0.9900\n",
            "Epoch 22/30\n",
            "172/172 - 5s - loss: 0.0028 - accuracy: 1.0000 - val_loss: 0.0162 - val_accuracy: 0.9933\n",
            "Epoch 23/30\n",
            "172/172 - 5s - loss: 0.0028 - accuracy: 1.0000 - val_loss: 0.0293 - val_accuracy: 0.9867\n",
            "Epoch 24/30\n",
            "172/172 - 5s - loss: 0.0062 - accuracy: 0.9988 - val_loss: 0.0994 - val_accuracy: 0.9700\n",
            "Epoch 25/30\n",
            "172/172 - 5s - loss: 0.0097 - accuracy: 0.9994 - val_loss: 0.0461 - val_accuracy: 0.9867\n",
            "Epoch 26/30\n",
            "172/172 - 5s - loss: 0.0039 - accuracy: 0.9994 - val_loss: 0.0135 - val_accuracy: 1.0000\n",
            "Epoch 27/30\n",
            "172/172 - 5s - loss: 0.0025 - accuracy: 1.0000 - val_loss: 0.0115 - val_accuracy: 0.9967\n",
            "Epoch 28/30\n",
            "172/172 - 5s - loss: 0.0047 - accuracy: 0.9988 - val_loss: 0.0452 - val_accuracy: 0.9767\n",
            "Epoch 29/30\n",
            "172/172 - 5s - loss: 0.0033 - accuracy: 0.9994 - val_loss: 0.0105 - val_accuracy: 1.0000\n",
            "Epoch 30/30\n",
            "172/172 - 5s - loss: 0.0015 - accuracy: 1.0000 - val_loss: 0.0283 - val_accuracy: 0.9900\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fe9e8131e10>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YD3lwqQxrRh-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# %cp -r \"/content/drive/My Drive/data_set_to_train/Sign-Language-Digits-Dataset/Dataset\" /content/"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aPP-MyTfuaT_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_labels = test_batches.classes"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LXu9ABpHx1Gh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predictions = model.predict(x=test_batches, verbose=0)"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0O71kxASx3Xg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_confusion_matrix(cm, classes,\n",
        "                          normalize=False,\n",
        "                          title='Confusion matrix',\n",
        "                          cmap=plt.cm.Blues):\n",
        "    \"\"\"\n",
        "    This function prints and plots the confusion matrix.\n",
        "    Normalization can be applied by setting `normalize=True`.\n",
        "    \"\"\"\n",
        "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
        "    plt.title(title)\n",
        "    plt.colorbar()\n",
        "    tick_marks = np.arange(len(classes))\n",
        "    plt.xticks(tick_marks, classes, rotation=45)\n",
        "    plt.yticks(tick_marks, classes)\n",
        "\n",
        "    if normalize:\n",
        "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
        "        print(\"Normalized confusion matrix\")\n",
        "    else:\n",
        "        print('Confusion matrix, without normalization')\n",
        "\n",
        "    print(cm)\n",
        "\n",
        "    thresh = cm.max() / 2.\n",
        "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
        "        plt.text(j, i, cm[i, j],\n",
        "            horizontalalignment=\"center\",\n",
        "            color=\"white\" if cm[i, j] > thresh else \"black\")\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.ylabel('True label')\n",
        "    plt.xlabel('Predicted label')"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W0t3xedzx8cP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cm = confusion_matrix(y_true=test_labels, y_pred=predictions.argmax(axis=1))"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B0uZ9Necx-lo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "ecb13a93-efa5-4538-c9ce-4b8eb987cbfa"
      },
      "source": [
        "test_batches.class_indices"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'0': 0,\n",
              " '1': 1,\n",
              " '2': 2,\n",
              " '3': 3,\n",
              " '4': 4,\n",
              " '5': 5,\n",
              " '6': 6,\n",
              " '7': 7,\n",
              " '8': 8,\n",
              " '9': 9}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-q5pizV-yAav",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 498
        },
        "outputId": "c983e461-a437-42cc-ec3d-45c4a6261d0f"
      },
      "source": [
        "cm_plot_labels = ['0','1','2','3','4','5','6','7','8','9']\n",
        "plot_confusion_matrix(cm=cm, classes=cm_plot_labels, title='Confusion Matrix')"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Confusion matrix, without normalization\n",
            "[[5 0 0 0 0 0 0 0 0 0]\n",
            " [0 5 0 0 0 0 0 0 0 0]\n",
            " [0 0 5 0 0 0 0 0 0 0]\n",
            " [0 0 0 5 0 0 0 0 0 0]\n",
            " [0 0 0 0 5 0 0 0 0 0]\n",
            " [0 0 0 0 0 5 0 0 0 0]\n",
            " [0 0 0 0 0 0 5 0 0 0]\n",
            " [0 0 0 0 0 0 0 5 0 0]\n",
            " [0 0 0 0 0 0 0 0 5 0]\n",
            " [0 0 0 0 0 0 0 0 0 5]]\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAT8AAAEmCAYAAAAUf5f4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZxcVZ338c83HQIEAgIJIIEICIIBDUtkVQQVDYvLOCgoooKKjuAKj4/OqKgjjsuoMI84GjfEsAkYFZHtUZFlFCERkFVlExIUguyLkPCbP85tKJru6lvVd6lb9X3zuq/Ucut7b4ruk3OXc36KCMzMBs2kunfAzKwObvzMbCC58TOzgeTGz8wGkhs/MxtIbvzMbCC58RswklaVdKak+ySdNoGcAyWdV+S+1UHS2ZLeVvd+WPXc+PUoSW+WdLmkByXdkf2SvriA6P2A9YB1IuIN3YZExIkR8coC9udpJO0uKSQtHPH6nOz1C3LmfErSgvHWi4i9IuL7Xe6uNZgbvx4k6cPAMcDnSA3VLODrwGsLiH8O8MeIWF5AVlnuAnaWtE7La28D/ljUBpT453+QRYSXHlqANYEHgTe0WWdlUuO4NFuOAVbO3tsduB04ArgTuAM4OHvv08BjwOPZNt4BfApY0JK9MRDA5Oz524GbgAeAm4EDW16/uOVzuwCXAfdlf+7S8t4FwL8Dl2Q55wHTx/i7De//N4DDsteGgCXAJ4ELWtY9FrgNuB9YBLwke33eiL/nlS37cXS2H48Am2WvvTN7/7+BM1ryvwD8AlDdPxdeil/8L1/v2RlYBVjYZp1/A3YCtgHmADsAH295f31SIzqT1MAdJ2mtiDiK1Js8NSJWj4jvtNsRSasB/wXsFRHTSA3cFaOstzZwVrbuOsBXgLNG9NzeDBwMrAtMAY5st23gBOCt2eNXAVeTGvpWl5G+g7WBk4DTJK0SEeeM+HvOafnMQcChwDTg1hF5RwAvkPR2SS8hfXdvi6wltP7ixq/3rAMsi/aHpQcCn4mIOyPiLlKP7qCW9x/P3n88In5O6v1s0eX+PAFsLWnViLgjIq4ZZZ19gD9FxA8iYnlEnAxcD7y6ZZ3vRcQfI+IR4IekRmtMEfE/wNqStiA1gieMss6CiLg72+aXST3i8f6ex0fENdlnHh+R9zDpe/wKsAB4X0TcPk6eNZQbv95zNzBd0uQ262zA03stt2avPZkxovF8GFi90x2JiIeA/YH3AHdIOkvSljn2Z3ifZrY8/2sX+/MD4HBgD0bpCUs6UtJ12ZXre0m93enjZN7W7s2IuJR0mC9SI219yo1f7/kN8A/gdW3WWUq6cDFsFs88JMzrIWBqy/P1W9+MiHMjYk/g2aTe3Ldy7M/wPi3pcp+G/QB4L/DzrFf2pOyw9CPAG4G1IuJZpPONGt71MTLbHsJKOozUg1ya5VufcuPXYyLiPtKJ/eMkvU7SVEkrSdpL0hez1U4GPi5phqTp2frj3tYxhiuA3STNkrQm8LHhNyStJ+m12bm/f5AOn58YJePnwPOy23MmS9ofmA38rMt9AiAibgZeSjrHOdI0YDnpyvBkSZ8E1mh5/2/Axp1c0ZX0POCzwFtIh78fkdT28Nyay41fD8rOX32YdBHjLtKh2uHAj7NVPgtcDlwF/AFYnL3WzbbOB07Nshbx9AZrUrYfS4G/kxqifxkl425gX9IFg7tJPaZ9I2JZN/s0IvviiBitV3sucA7p9pdbgUd5+iHt8A3cd0taPN52stMMC4AvRMSVEfEn4F+BH0haeSJ/B+tN8oUsMxtE7vmZ2UBqd0XRzKwxJN1Cuol+BbA8Iua2W9+Nn5n1kz3ynmv2Ya+ZDaSeuuChyauGpkwrJXvb588qJddsENx66y0sW7ZM46+Z39Aaz4lY/kju9eORu64hXdUfNj8i5g8/kXQzcA/pXs5vtr43mp467NWUaay8xRtLyb7k0q+Vkms2CHbdse3ps67E8kc6+n1/9IrjHh3nPN6LI2KJpHWB8yVdHxEXjrWyD3vNrCYCTcq/jCMilmR/3kkaDrlDu/Xd+JlZPQRI+Zd2UdJqkqYNPwZeSZoJaEw9ddhrZgOmuPlk1wMWKjWSk4GTsqnNxtTInt/1Z32ay374r/z2lI9y8YnFjj0/79xzeOFWW7DVlpvxpS9+3tklZ5ed7+xqszsjmDSUf2kjIm6KiDnZslVEHD3e1hvZ+AHMO/RYdjrg87z4wC+Ov3JOK1as4IPvP4yfnHk2v7/qWk475WSuu/ZaZ5eUXXa+s6vN7kpBh73daGzjV4bLfvc7nvvczdhk002ZMmUKb9j/AH525k+cXVJ22fnOrja7Y6LQCx6damTjFxGc+fXDueTEj3DI63ctLHfp0iVsuOFGTz6fOXNDliyZ6JR0zq4r39nVZneug15fCT2/Ui94SJpHKjIzBHw7Igo5wfDyg7/K0rvuY8Zaq/OzbxzODbf8lUsW31hEtJlVqcYCeqVtWdIQcBywF2liyzdJml1E9tK77gPgrnse5Ke/vIoXbbVxEbFssMFMbr/9qSnhliy5nZkzZ7b5hLN7Od/Z1WZ3pU/P+e0A/Dm7CvMYcAoF1J2dusoUVp+68pOPX7HzllxzY7czuD/d3Be9iD//+U/ccvPNPPbYY5x26inss+9rnF1Sdtn5zq42u3PF3uTcqTIPe2fy9Jl1bwd2HLmSpENJpQRhpfFr2qy7zjRO/cq7AJg8NMSpZ1/O+f9zXQG7C5MnT+arx36NV+/zKlasWMHb3n4Is7faytklZZed7+xqszs2fJNzTUqb2EDSfsC8iHhn9vwgYMeIOHysz0yaum6UNbb3nss8ttesW7vuOJdFiy4vtKWaNG2DWHnbQ3Ov/+hFn1403hx9nSiz57cE2Kjl+YZMvJqXmfUNwVD7m5fLVOY5v8uAzSVtImkKcADw0xK3Z2ZNUvN9fqX1/CJiuaTDSVW2hoDvRsQ1ZW3PzBqoxnN+pd7nFxE/J9V0NTMbQbXe5+dZXcysPv3a8zMza8s9PzMbOCWN3MjLjZ+Z1cc9PzMbSO75Jds+f1ZpVdbWetGYA0smzKNHzLrhq71mNojEuNPTl8mNn5nVxD0/MxtUPudnZgOpH2dyLlOZpfdcFrPa7LLznV1tdsf6dCbnUlRRes9lMavJLjvf2dVmd0z1zuTcuMavp0rvdaCp5QhdutLZpXLPL7+yS++5LGZ12WXnO7va7G5Iyr0UrbQLHpK+C+wL3BkRW5e1naK5LKZZNVIJj/qu9pbZ8zsemFd0aNml91wWs7rssvOdXW12xyQ0Kf9StNIav4i4EPh70blllt5zWcxqs8vOd3a12d3oy8PevFpLV240a9a465dZes9lMavNLjvf2dVmd6POw97SSlcCSNoY+Fnec37bbz83Lrn08lL2xRMbmHWvjNKVQ2tvEqu/6jO517//lLc2pnSlmdnYlC01ceNnZrUQ5ZzLy6u0Cx6STgZ+A2wh6XZJ7yhrW2bWTH15wSMi3lRWtpn1hzp7fj7sNbPauPEzs8HjCx5mNoiEmDTJMzmb2QAq+rBX0hBwObAkIvZtt27jZnUxsz6iDpZ8PgDkGpY1MD2/MkdhlDl6BDyCxPqUiu35SdoQ2Ac4GvjweOsPTONnZr2nw8ZvuqTW8a/zI2J+y/NjgI8A0/KEufEzs9p02PgtG2tsr6ThuUMXSdo9T5gbPzOrRcHD23YFXiNpb2AVYA1JCyLiLWN9wBc8zKw+BV3wiIiPRcSGEbExcADwy3YNHzS08WtqWT+Xxaw+39nVZndE9Y7tbVzj1/Syfi6LWV2+s6vN7kYZjV9EXDDePX7QwMZvoMr65dTk76Sp++7sYvRlDY+yNLmsn8tiVpvv7Gqzu9GXU1pJ2gg4AVgPCNI9OceWtb0mcFlMs6eU1ajlVWbPbzlwRETMBnYCDpM0e6KhTS7r57KY1eY7u9rsbvTlBY+IuCMiFmePHyCNt5vwt9zUsn4ui1l9vrOrze5GXx72tsqquG0LXDrKez1TutJlMavNLjvf2dVmd6XG+fxKLV0JIGl14NfA0RHxo3brllm6skye2MD6XRmlK1deb/OYeWD+ywA3f3Wf5pSulLQScAZw4ngNn5kNmIJndelUmVd7BXwHuC4ivlLWdsysmQTU2PaVerV3V+Ag4GWSrsiWvUvcnpk1ipg0Kf9StDJLV15MraczzazX9eVhr5lZW6r3sNeNn5nVQlDK4WxebvzMrDbu+ZnZQPI5PzMbPD7nZ2aDKN3n555fo5U9/KzM4XMeOmf1qXdKKzd+ZlYbH/aa2eCRb3UxswFU9zm/xtXwgOaW9XNZzOrznV1tdqek/EvRGtf4NbWsn8tiVp/v7Gqzu9GX09iXpall/XqtZGBeLl3p7DK559eBppb1c1nM6vOdXW12x9SnNTwkrQJcCKycbef0iDiqrO0NOpfFtKbp58lM/wG8LCLmANsA8yTtNNHQppb1c1nM6vOdXW125/L3+hp1zi+SB7OnK2XLhKslNbWsn8tiVp/v7Gqzu1HnOb+yCxgNAYuAzYDjIuIZpSs71dSyfi6LWX2+s6vN7ljNNzmXXroSQNKzgIXA+yLi6hHvtdbt3f6PN95a+v40jcf2Wt3KKF05baMtY5sPfjv3+hcf+ZJCS1dWcrU3Iu4FfgXMG+W9+RExNyLmzpg+o4rdMbMe0Zfn/CTNyHp8SFoV2BO4vqztmVnz9Os5v2cD38/O+00CfhgRPytxe2bWMH05pVVEXAVsW1a+mTWcZ3I2s0EkT2ZqZoPKPT8zG0iT3PMzs0FUVNvXzVwCbvzMrBYSDBU3wmN4LoEHJa0EXCzp7Ij47VgfcONnZrUp6oJHpKFqHc0l4MavAcocguahc1anDtu+6ZIub3k+PyLmP5XV2VwCYzZ+kv4fbVrOiHh/7l02MxtBpNtdOrCs3djeiFgBbDM8l4CkrUfOJdCqXc/v8jbvmZlNWBmTukTEvZKG5xLovPGLiO+3Ppc0NSIeLm4XzWygFThhgaQZwONZwzc8l8AX2n1m3IkNJO0s6VqySQkkzZH09SJ22MwGW4ETGzwb+JWkq4DLgPPHm0sgz6wuxwCvAu4GiIgrgd1yfK40Ta1p2tRsaG5dYGdXm90JkW5yzru0ExFXRcS2EfHCiNg6Ij4z3vZzTWkVEbeNeGlFns+Voak1TZua3appdYGdXW12N3q9dOVtknYBQtJKko4EipkjvQtNrWna1OyyNfV7cXYxen0y0/cAhwEzgaWkSmyHFb4nOTW1pmlTs4c1sS6ws6vN7tTwCI+8S9HGvck5IpYBB3a7gezGw8uBJRGxb7c5Vi/XBbYy1DipS66rvZtKOlPSXZLulPQTSZt2sI0PUOBhclNrmjY1e1gT6wI7u9rsbvT6Ye9JwA9Jl5I3AE4DTs4TLmlDYB8gf4mmcTS1pmlTs6G5dYGdXW12p9LV3vxL0fKM7Z0aET9oeb5A0v/JmX8M8BFg2lgrjChdOW5gU2uaNjUbmlsX2NnVZnespB5d7s2PVbdX0trZw/8L3AOcQhrruz+wVkR8rG2wtC+wd0S8V9LuwJHjnfPbfvu5ccmlHlVXJU9sYHmUUbd3nU23ir3//aTc6y94yzaF1u1t1/NbRGrshv/C7255L4C2jR+wK/AaSXsDqwBrSFoQEW/pdmfNrL/0ZA2PiNhkIsFZz/BjAC09Pzd8ZgY8dc6vLrnm85O0NTCb1IMDICJOKGunzGww9GTPb5iko4DdSY3fz4G9gIuB3I1fRFwAXNDNDppZf5JgqMbGL8+tLvsBLwf+GhEHA3OANUvdKzMbCHWO7c1z2PtIRDwhabmkNYA7gY3G+5CZ2Xh6+rAXuDybFvpbpCvADwK/KXWvzGwg9HTR8oh4b/bwG5LOAdaIiKvK3S0z63di/Hn6ytSugNF27d6LiMXl7JKZDYSSzuXl1a7n9+U27wXwsoL3xWrgsphWp5485xcRe1S5I2Y2eHJNJV8SFy03s1qIHu35mZmVreeHt5mZFW14Gvu65JnJWZLeIumT2fNZknYof9fG1tSyfs4encti9k92p+qczDTP+cavAzsDb8qePwAcV/yu5NPUsn7Obs9lMZuf3Y1eL125Y0QcBjwKEBH3AFOK35V8mlrWz9nVa+r30tTsThVZtLwbeRq/x7MKbAEgaQbwROF7klNTy/o5e2wui9kf2d2Y1MFStDwXPP4LWAisK+lo0iwvH88TLukW0mHyCmB5kVNQW/9wWczB1asjPACIiBMlLSJNayXgdRHRSfWaPbLav4Voalk/Z49ttLKYRTR+Tf1emprdKZV0OJtXnqu9s4CHgTOBnwIPZa/Voqll/Zw9OpfF7J/sbvT6fH5n8VQho1WATYAbgDz17gI4T1IA34yI+SNXcOnKwc0Gl8Xsp+xu1HmT85ilK8f8QJrt5b0R8c4c686MiCWS1gXOB94XEReOtb5LV/YXT2zQP8ooXTnzeS+I93x9Ye71P7nn5oWWruz4Iko2ldWOOdddkv15J+miSa03R5tZD+ngBucyeoh5Chh9uOXpJGA7YNyTMpJWAyZFxAPZ41cCn+l2R82s/4j6jnvznPOb1vJ4Oekc4Bk5PrcesDCbtWEycFJEnNPxHppZX+rpur3Zzc3TIuLIToMj4iZSpTczs1H1ZOMnaXJELJdU3C33ZmYtenU+v9+Rzu9dIemnwGnAQ8NvRsSPSt43M+tjPX3Ym1kFuJtUs2P4fr8A3PiZWfcKvHlZ0kbACaRrDQHMj4hj232mXeO3bnal92qeavSGdXZzoJnZKAoc3rYcOCIiFkuaBiySdH5EjDlfV7vGbwhYHUa9Fu3Gz8wmpMjD3oi4A7gje/yApOuAmUBXjd8dEeH78qxrLotp7Ymhznp+0yW1DgGbP8aQ2Y2BbYFL24W1a/xqPBVpZv0uVW/r6CPLxhveJml10n3IH4yI+9ut267xe3lHu2Vm1omCh61JWonU8J2Y526UdkXL/17cbpmZPVNRFzyUbhj8DnBdRHwl17YL2bKZWYeGD3sLms9vV+Ag4GWSrsiWvdt9oJGNX1PL+jm7+nyXxaw2u1NFFTCKiIsjQhHxwojYJlt+3nbbhf5NKtDUsn7OricfXBazquxu9Hrpyp7S1LJ+zq4nvyxN/c576fsW9VZva1zj19Syfs6uJ99lMavL7pjSxAZ5l6LlGdvbNUnPAr4NbE0aFXJIRPymzG2atXJZzN5W583EZff8jgXOiYgtSXP7TbgyTVPL+jm7nvzRymIWoanfeU+VrgSGpNxL0Upr/CStCexGuveGiHgsIu6daG5Ty/o5u/p8l8WsNrsbvV66slubAHcB35M0B1gEfCAiHmpdyaUrBze77HyXxaw2u3PlnMvLvfVOS1fmDpbmAr8Fdo2ISyUdC9wfEZ8Y6zMuXWl5eWKDapVRuvK5s+fE505seyve0xyw3Yb1lq7swO3A7RExPLPC6aSZoc3MgHqv9pbW+EXEX4HbJG2RvfRy2sytZWaDRx0sRSv1VhfgfcCJkqYANwEHl7w9M2sK9W4BowmLiCuAwo7Rzax/DI/wqEvZPT8zszH1bc/PzKydXi9daWZWuHTY656fmQ2gGo963fiZWV2E3PMz60xTy2KCR5C0cs/PzAaOz/mZ2WAqabaWvNz4mVlt3PiZ2UCq84JH42p4QHPL+jm7+nyXxaw2uxMi3eScdyla4xq/ppb1c3b1+S6LWW12N4qq29vVtgtPLFlTy/o5u/r8XirT2IlB+k7UwX9Fa1zj19Syfs6uPt9lMavN7lTdh72lXfDIJjE9teWlTYFPRsQxZW3TrEouizlR9Y7wKHMm5xsiYpuI2AbYHngYWDjR3KaW9XN29fkui1ltdsc6qNxWxi0xVR32vhy4MSJunWhQU8v6Obv6fJfFrDa7G/08jf2wA4CTR3vDpSsHN7vsfJfFrDa7U+mcXx+WrnxyA6l+x1Jgq4j4W7t1XbrSeoEnNnimMkpXPv8F28b3Fv4q9/o7b75WoaUrq+j57QUsHq/hM7MB1OfD297EGIe8ZjbY6jzsLfWCh6TVgD2BH5W5HTNrpr694BERDwHrlLkNM2uwPj/sNTN7htSj82SmZjZoPJmpmQ2qGtu+5k1sYGZ9pMArHpK+K+lOSVfn2bQbPzOrSScTWuXqIx4PzMu7dR/2mlltijznFxEXSto47/pu/MxGKHv4WZnD55o0dK6L+/emS2od/zo/IuZ3u303fmZWG3XW9VvWtLG9Zmaj8q0uZjaQfKtLh5pa1s/Z1ec3NbupZTE70sltLvludTkZ+A2whaTbJb2j3fqN6/kNl9476+zzmbnhhrx4pxex776v4fmzZzu7Ydll5zc1e9i8Q4/l7nsfKiwPqtnvThQ5vC0i3tTJ+o3r+TW1rJ+zq89vanaZemm/xWDU8ChMU8v6Obv6/KZmQzPLYnajb6e0kvQh4J1AAH8ADo6IR8vcplk/GJiymDVe8Sit5ydpJvB+YG5EbA0MkQoZTUhTy/o5u/r8pmZDM8tidqPg4W0dKfuwdzKwqqTJwFRSIaMJaWpZP2dXn9/U7KaWxezGJOVfilbaYW9ELJH0n8BfgEeA8yLivJHruXTl4GaXnd/U7KaWxexKjYe9pZWulLQWcAawP3AvcBpwekQsGOszLl1pg6CJY3vLKF35gjnbxY/OuyT3+s9bf2qhpSvLPOx9BXBzRNwVEY+TihjtUuL2zKxJOrjNpWm3uvwF2EnSVKXRyy8Hium7m1lf6MtbXSLiUkmnA4uB5cDvga6nnzGzPtSvExtExFHAUWVuw8yaqpxbWPJq3NheM+sfntLKzAZOWefy8nLjZ2b1cc/PzAbRpBqPe934mVltfNhrZoOnpJuX83LjZ1axMstLljV07h83/KWU3Dr7fm78zKwWwzM518WNn5nVxuf8zGwg1dnza1wND2huOUJnV5/v7Gcqsyxmp/p5JufCDZfe+8mZZ/P7q67ltFNO5rprr3V2A7PLznf22OYdeiw7HfB5XnzgFwvN7ViN07o0rvFrajlCZ1ef7+zeV+eUVo1r/JpajtDZ1ec7e3RllcXslJRGeORdilZ26coPAO8iNdzfiohjytyemY2vp8pi9uMFD0lbkxq+HYA5wL6SNptoblPLETq7+nxnj66sspjd6NfD3ucDl0bEwxGxHPg18PqJhja1HKGzq8939jOVWRazG3XW8CjzsPdq4GhJ65BKV+4NTLg0W1PLETq7+nxnP1OZZTE7V+9MzqWVrgSQ9A7gvcBDwDXAPyLigyPWaa3bu/0fb7y1tP0x63flje39IU88fGehLdW2282NX158ae71115tcmNKVxIR34mI7SNiN+Ae4I+jrDM/IuZGxNwZ02eUuTtmZk8q+2rvuhFxp6RZpPN9O5W5PTNrln6e2OCM7Jzf48BhEXFvydszswbp2+ptEfGSMvPNrLnSTc71bd+zuphZfdz4mdkg6tvDXjOzdjyfn5kNpCKHt0maJ+kGSX+W9NHx1nfjZ2b1Kaj1kzQEHAfsBcwG3iRpdrvPuPEzs9oUOJPzDsCfI+KmiHgMOAV4bbsP9NQ5v8WLFy1bdSXlHd82HVhW0q6UmV12vrP7J7vs/E6yn1P0xn+/eNG5U6doegcfWUVS6/wA8yNifvZ4JnBby3u3Azu2C+upxi8ico9vk3R5keP8qsouO9/Z/ZNddn7Z+z6eiJhX17bBh71m1h+WABu1PN8we21MbvzMrB9cBmwuaRNJU4ADgJ+2+0BPHfZ2aP74q/Rkdtn5zu6f7LLzy973ykTEckmHA+cCQ8B3I+Kadp8pdT4/M7Ne5cNeMxtIbvzMbCC58bNcpDpHYXZH0molZq/fxO/EntKoxk/SFpJ2lrRSNpyl6PzCM7PczSTNlbRyCdlbSXppNmls0dkvlnQQQERE0b/skl6d1XYunKTXAl+QtG4J2a8CFvL0WyuKyt5J0kHZn1MKzt48+zmcVNbPeqNERCMW0jT41wO/AE4A3g+sUVD281oeDxW83/sCVwG/Ak5u3VYB2Xtl2T8GzgLWLyh3ErA6qejUtcB7Wt8raBuvBK4A9izhZ+Wl2c9KGdnD+30LcGzB2a/J/n9+Hzgd2LzA7NcBVwJnAMeQCoutVvT306Sl9h3I+T9uJeBUYNfs+T8DXwKOnmgDmDVODwMntbxWSAMI7AJcB2ybPf866RJ8Edm7kwpC7ZA9Xwi8ouDv/SPAEdk/Nh8qMHcX4G8t+74mafjU1ILyPwwcmT3eANiTNNRpzQnmvgL4M7BV9jN5HrBbQfu8Duk2ja2z598F3gCsC6xSQPbZwOzs+SGk++I+AUwr8memSUuTDnvXADbPHi8Efkb6AXxzt4dj2Tmhw4EPAo9JWgAQESsKPCz4QkT8Pnt8FLB2QYe/fwPeHRG/k7Q+6Zf7cEnflLRfQYeoy0mHdt8HdpD0FUn/oWQiPzt3k+q6PDs7XP8x8N/A8QXt+/KWx6eTftkPB46TtNYEcoeAt0a6f2w14AZSQ1jEOdHlwKrAlpLWIP3j9lZSL+3jEzx/uZzUk18fICK+S+q5Tif94z+Y6m59O/jXa0/SHdsvyZ4PAW8GFpDdr9hl7gakH4zppF+UBQXu8xBZzzR7vCHwe2BG9to6BW3n34CPZ4/fTprRYkYBuc8FPpo9PoLUQz6uoH2eA9xEGoD+LtKh9iGkUwNrTzD7BaSG6RTg4Oy1TYFvAK8qYN8nZX/OA/4KvKCg72Q/YBHwW+AT2WsvA44H5kww+z3Z78pBpCOmBcC7ge8Use9NXJrU87uIdJhxkKTdImJFRJxEarzmdBsaEUsj4sGIWEb6YVh1uAcoaTtJW04ge0VE3J89FXAv8PeIuEvSgcBnJa3abX7Ldo6OiM9mj48n9ZKLOBn/CLCFpHeRfnk+D8yS9O6JBkfElaRex+cj4lsR8USkHslawKwJZv8BOJLUG94ke+0m0j9AEy4OHRFPZH+eQxolsW8BvWEi4nTSofVFpH8kiYhfAtOY+KwqJ5MOffcAVo2It0TEN4H1sp7mwGnM8LaIeFTSiUAAH8sapX8A6wF3FLSNu7Nf7C9Jup70y7JHQdnLgQcl3SbpP0gnzt8eEY9MJFeSIvunPXv+z6TvZOmEdpj0D32Z0A0AAARiSURBVIOk20jnhg6LiDMl7UE67zVhEXEt6YIK8OS+z6CY/59nk04zfEp6cpq0bUkNeJGuBD4EfDEiVkw0LCLukfRL4I2SHgNWITXgV00w9z7gREknDzfekt4KrA1MeL8bqe6uZ6cLMIXUIJ1COhzYtoRtfIgCD2eyTGX7fiPwFwq8kpflrwy8g3SFdusCczcCtm95XsjV3lG+m0NIDeFWBWdvB3wO+HKR/z9HbOOHwMYF5j2LdDfDr0kXQSZ0yDvGNoa/71K+kyYsjR3bm12QiMj+FSswdy3SD/MRETGhf23HyH87cFmMM+i6i9yVSOdFb4yIG4rMzvKf1sMsOpt0e8pfI+L6MrZRhjK/kyx/Gul89v3jrtx59nOAlSKikF58EzW28SuTpFUi4tGSskv9hTGzfNz4mdlAatLVXjOzwrjxM7OB5MbPzAaSGz8zG0hu/PqEpBWSrpB0taTTJE2dQNbxkvbLHn9bbSrfS9pd0i5dbOMW6Zk1W8d6fcQ6D3a4rU9JOrLTfbT+5savfzwSEdtExNbAY6ThaE+S1NVonoh4Z6SRGGPZnTRLi1mjuPHrTxcBm2W9sosk/RS4VtKQpC9JukzSVcNjdLNxqV+TdIOk/0+aRonsvQskzc0ez5O0WNKVkn4haWNSI/uhrNf5EkkzJJ2RbeMySbtmn11H0nmSrpH0bdKojrYk/VjSouwzh45476vZ67+QNCN77bmSzsk+c9FExmVb/2vM2F7LJ+vh7QWck720HWm4281ZA3JfRLxIaVqtSySdRxrzugUwmzQu+FrSfHKtuTOAb5Hmr7tZ0toR8XdJ3wAejIj/zNY7CfhqRFwsaRZpeNbzSeNsL46Iz0jahzQUbzyHZNtYFbhM0hkRcTdpOqnLI+JDkj6ZZR9OmmTgPRHxJ0k7kuZPfFkXX6MNADd+/WNVSVdkjy8CvkM6HP1dRNycvf5K4IXD5/NIk4huDuwGnBxpYP7SbGD9SDsBFw5nRcTfx9iPVwCz9dT0dmtIWj3bxuuzz54l6Z4cf6f3S/qn7PFG2b7eDTxBmtwW0tRMP8q2sQtwWsu2Cy8bYP3DjV//eCQitml9IWsEHmp9CXhfRJw7Yr29C9yPScBOI4cHqsO5PiXtTmpId46IhyVdQJrhZDSRbffekd+B2Vh8zm+wnAv8SzYJApKepzRD8IXA/tk5wWcz+jRevwV2k7RJ9tm1s9cfIM03N+w84H3DTyQNN0YXkiafRdJepHn72lkTuCdr+LYk9TyHTSJN/EmWeXE2+P9mSW/ItiFJXc/zaP3Pjd9g+TbpfN5iSVcD3yT1/hcCf8reOwH4zcgPRsRdwKGkQ8wreeqw80zgn4YveJCmYpqbXVC5lqeuOn+a1HheQzr8/cs4+3oOMFnSdaQ5+H7b8t5DpGn1ryad0/tM9vqBwDuy/bsGeG2O78QGlCc2MLOB5J6fmQ0kN35mNpDc+JnZQHLjZ2YDyY2fmQ0kN35mNpDc+JnZQPpfbZCtJsP/zXEAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BaJ5ReiCyEns",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 41,
      "outputs": []
    }
  ]
}